#-------------------------------------------------------------------------------
# Name:        module1
# Purpose:
#
# Author:      Kim
#
# Created:     26/11/2011
# Copyright:   (c) Kim 2011
# Licence:     <your licence>
#-------------------------------------------------------------------------------
#!/usr/bin/env python

from DBPediaProcess import *
from wordProcessing import *
import urllib, HTMLParser
import os

SPOTLIGHT_ADDR = "http://spotlight.dbpedia.org/rest/annotate"
SPOTLIGHT_SETTINGS = "&confidence=0.1&support=100"


class spotlightParser(HTMLParser.HTMLParser):
    def __init__(self):
        HTMLParser.HTMLParser.__init__(self)
        self.link = ""
        self.word = ""
        self.wordMap = {}

    def handle_starttag(self, tag, attrs):
        if (tag == "a"):
            self.link = attrs[0][1]

    def handle_endtag(self, tag):
        if (tag == 'a'):
            if (len(self.word) > 0 and len(self.link) > 0):
                self.wordMap[self.word] = self.link
            self.word = ""
            self.link = ""

    def handle_data(self, data):
        if (len(self.link) > 0):
            self.word = data

    def getData(self):
        return self.wordMap

class WordRelationProcessor():
    def __init__(self):
        # stopwords
        stopwordsfile = open('stopwords.txt')
        self.stopwords = stopwordsfile.next().strip().split(',')
        stopwordsfile.close()

        self.subject_data = {}

    def QuerySubjects(self, page):
        query = Query()
        query.addSelect("?subject")
        query.addWhere(page, "<http://purl.org/dc/terms/subject>", "?subject")
        queryString = query.getQueryString()

        results = DBPediaQuery(queryString)
        if (results == None):
            print "ERROR: Failed to query in DBPedia"
            return None
        trueResults = results['results']['bindings']
        if len(trueResults) == 0:
            print "ERROR: Empty TrueResults"
            print results
            print "\n\n"
            return None
        else:
            subjects = []
            for eachResult in results['results']['bindings']:
                subject = eachResult['subject']['value']
                subjectTuple = (subject, 10)
                subjects.append(subjectTuple)
            return subjects

    def addSubjectCandidate(self, subject, score):
        key = None
        for eachKey in self.subject_data.keys():
            if eachKey.find(subject) >= 0:
                key = eachKey
        if key == None:
            self.subject_data[subject] = score
        else:
            cur_score = self.subject_data[key]
            self.subject_data[key] = cur_score* 2

    def processSubjectFromDBPedia(self, subjectString):
        subjects = []
        words = getSubjectFromPage(subjectString)
        if (words.find("category:") == 0):
            words = words[9:]
        words = words.split("_")
        for word in words:
            word = word.strip(",")
            word = word.replace("%28", "")
            word = word.replace("%29", "")
            try:
                number_string = int(word[0])
                continue
            except ValueError:
                pass
            if (not (word in self.stopwords)):
                subjects.append(word)
        return subjects

    def TopicWord(self, words):
        words = words.replace(":", "")
        url_string = urllib.pathname2url(words)
        addr = SPOTLIGHT_ADDR + "?text=" + url_string + SPOTLIGHT_SETTINGS
        fp = None
        count = 0
        errorString = ""
        while (fp == None and count < 5):
            try:
                fp = urllib.urlopen(addr)
            except Exception as e:
                errorString = str(e)
                count = count + 1

        if (len(errorString) > 0):
            print "ERROR: Failed to read Spotlight: " + errorString
            return ("", "")

        if (fp == None):
            print "ERROR: Failed to get the contect for "
            print addr
            print "\n"
            return ""

        parser = spotlightParser()
        parser.feed(fp.read())
        fp.close()

        pages = parser.getData()

        for eachWord in pages.keys():
            page = pages[eachWord]
            page = "<" + page + ">"

            self.addSubjectCandidate(eachWord, 1000)

            subjects = self.processSubjectFromDBPedia(page)
            if subjects <> None:
                for eachSubject in subjects:
                    self.addSubjectCandidate(eachSubject, 100)

            subjects = self.QuerySubjects(page)
            if subjects  == None:
                continue

            subjectCollection = []
            #Add them to the new topic Word Sets
            for eachSubject in subjects:
                subject = eachSubject[0]
                score = eachSubject[1]
                processedSubject = self.processSubjectFromDBPedia(subject)
                subjectCollection = subjectCollection + processedSubject
            #Combine them into sets so that it doesn't get skewed for multiple subjects
            subjects = set(subjectCollection)
            for eachSubject in subjects:
                self.addSubjectCandidate(eachSubject, 10)

        subject_set = []
        for subject in self.subject_data.keys():
            subject_set.append((subject, self.subject_data[subject]))
        subject_set = sorted(subject_set, key=lambda subject: subject[1], reverse=True)

        if (len(subject_set) == 0):
            return ""

        return subject_set

def getTopTopicWord(wordString):
    myProcessor = WordRelationProcessor()
    topic_word = myProcessor.TopicWord(wordString)
    if topic_word:
      return topic_word[0][0]
    else:
      return None

if __name__ == '__main__':
    try:
        words = sys.argv[1]
    except Exception as e:
        print "usage: python TopicWordsDBPedia.py \"[WORDSTRING]\""
        print "ex. python TopicWordsDBPedia.py \"Chicago, Bears, Navy Pier\""
        sys.exit(0)


    myProcessor = WordRelationProcessor()

    print "Find Topic Words for " + str(words)
    topic_word = myProcessor.TopicWord(words)
    print "Topic Word: "
    print topic_word
